{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sklearn.metrics使用大全\n",
    "参考[官网资料](https://scikit-learn.org/stable/modules/classes.html#module-sklearn.metrics)，[相关算法](https://scikit-learn.org/stable/modules/model_evaluation.html#classification-metrics)。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入相关的包\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "        bird       0.00      0.00      0.00         1\n",
      "         ant       0.67      1.00      0.80         2\n",
      "         cat       0.67      0.67      0.67         3\n",
      "\n",
      "    accuracy                           0.67         6\n",
      "   macro avg       0.44      0.56      0.49         6\n",
      "weighted avg       0.56      0.67      0.60         6\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 输出精确和召回\n",
    "from sklearn.metrics import classification_report\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")  # 忽略当某个类别某有预测到时出的警告\n",
    "\n",
    "y_true = [\"cat\", \"ant\", \"cat\", \"cat\", \"ant\", \"bird\"]\n",
    "y_pred = [\"ant\", \"ant\", \"cat\", \"cat\", \"ant\", \"cat\"]\n",
    "labels=list(set(y_true))\n",
    "print(classification_report(y_true, y_pred,labels=labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上表中的参数解释\n",
    "\n",
    "|参数|解释|\n",
    "| ------------ | ---- |\n",
    "| precision          | 精确率 |\n",
    "| recall         | 召回率 |\n",
    "| f1-score          | f1 值 |\n",
    "|    support          |    总数  |\n",
    "| accuracy     |   准确率   |\n",
    "| macro avg    | 宏平均 |\n",
    "| weighted avg | 加权平均 |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.         0.66666667 0.66666667]\n",
      "[1.         0.         0.66666667]\n",
      "0.6666666666666666\n",
      "[0.8        0.         0.66666667]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score,recall_score,accuracy_score,f1_score\n",
    "\n",
    "print(precision_score(y_true, y_pred, average=None))\n",
    "print(recall_score(y_true, y_pred, average=None))\n",
    "print(accuracy_score(y_true, y_pred))\n",
    "print(f1_score(y_true, y_pred, average=None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1, 'Confusion matrix,without normalization')"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW8AAAEbCAYAAAAVjwXDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3debgcVZnH8e/vJoRNFiWbJkDYBsIiE43smAgyREBA2XEEFIwwooIwLug4GRyYYQQRCaAREGFAdIjIDsoSUDYFM4aYgCCyhJAQRlnCFpK888epGzpN3+7q3N6q8/s8Tz23a+lTb1f3ffv0qVOnFBGYmVmx9LQ7ADMzq5+Tt5lZATl5m5kVkJO3mVkBOXmbmRWQk7eZWQE5eWckTZI0X1JIOqoB5Y3KyhrbgPA6nqRpkia3cH+TJM2ssc1K9R40i6Tx2XEcXGm+iftt6WeqaDo6eUsaJukcSX+W9IakZyTdJGmvBu9na+BfgWOBdwM/bUCxT2dl/W8Dymq5FUh8Hwe+1syYypwJjOudkXSJpOtbuP/lrGSJ5h7SZ/v/GlGYpKMkLaywqtWfqUIZ2O4A+iJpFHA38DLpDfwD6ctmd+D7wAYN3N2m2d9fRIOuWoqIJcC8RpTVySQNiohFEfHXVu43IhYClf7hVzqSegBln7mmi4hFtOCz3erPVOFEREdOwI3AXOAdFda9s+TxBsDVpCT/MvBzYGTJ+knATOBQ4M/ZNr8ABpesj9IpW34JcH3ZficBM0vmtwFuA17Kyv0D8KFs3aisvLEl238QuB94HZgPnA0MKlk/DTgfOB14HniOVMPsqXKcjiIlsY8ADwOvAtcC6wAHAo8CLwKXAauXPG8C8Gvgb8BfgVuA0SXro2yaVnpcgK8Ac4DnSmKfnD3eHHgFOKJsf4uAHfp4HT8FLiiZPy3b7/Yly+YAnyh/Lyq9h8D4kvfgAOBX2bGZBexRtu8878vksucs+3xkj8v3P6qP11nzPQbeCfw4e29eA24Ftqrwnu9F+mwvBrYue2/mZe/7f5IqPZOyfc0DvlIW05eAGdl79gxwIbBuyfrx2Wsa3Mf8ExVe/7JjUK38krJKp0mVjnsdx2X37Li8AtwBbNTufNaMqSObTSS9i/TPPjlSDWs5EfG3bDuREvEwYDfgQ8B7gF9k63qNAg4BPgb8AzCGlBwg/eN8Jnv87mzK6wrgWWC7rMxJpARQ6TWNAG4CpmfbHg0cBvxH2aafIP0z7gQcD5yQxV7NqsBJ2XN3B8YCVwFHkhLX/sA+wD+VPGdN4LtZ7ONJ/+jXSRqUrd8u+zuBdEw+XvLcccB7s3W7lwcTEY8AJwKTJW0iaQgpsZwWEff18Rqmkd6/XuNJye1DAJI2A0Zk25U7E/gZ6Z+59z28p2T9acD3gG2B3wFXSnpHVm7e96WaLwL3Aj8q2f/TVbav9R5fAmwP7Ed6H14Fbpa0esk2qwHfAD4LbAk8mS3/ILAR6fgdC3yZVBFaFdiF9Bn9T0nvLylraRbDVsDh2T7PzfXKkw/w1ut+N+kL5GHSF2Gt8u/J1r1a8vwz+9jPJdQ+LquSfql/GtgRWJf0S737tPvbo4/ayXakb+CP1dhuD2AJJbUcYGPSh+XD2fwkUkJdp2SbrwOPlcwfSFbjLll2CbVr3i8BR/YR2yhKat6kBPIYy9ewjgLeANaIt2oa95aV8yvgwirH4KhsP5uXLDszOy6Dq72esnLWzJ6zS6X4y8pZAKxatnwab6+dXg3cB9xAagIbUGX/o7P9vRtYIzsuXwVuydZ/Bni0yntR6f3qfQ2fLVk2IlvW+zrzvi991rz72qaP11n1PQY2y+L7YMn6dUhfrseUvefvrxDT06XHGXgAmFG23RPAyVVinJC9/p5sfjxVat5lz/0K6Ut3kzrKPwpY2Mex6v01V89xKf1f+ATpF1+fv16LOnVkzRtQ7U2A9A8/NyKe6F0QEY+Tmlu2LNnuyYh4sWR+LjC0v0EC3wEulHS7pK9L2qJGrPdGxNKSZb8BBvFWmzukn5el8sT6RqTabq/5wLyIeL5s2bJyshrxFdnJ4Jey9T3kO5cwMyLeyLHdMaTX9kHgH6NKm2xEzM5iGA/sTGriuhLYWdIq2fJpOfZZSekxnZv97T0Wed+XRqr2Ho8mVT7u7V2ZfXYfYvnP9GIqnwyfVXac52fPpWxZ6WdhN0m/kjRHUm/T4yBgeO5XlMr5KPBvwAER8ecGl5/3uJT/L8wFViHVwLtKpybvR0nfoKNrbKdsu0pKl79ZYV2t176Ut3+JrLJcIRGTSB+cX5B+As+Q9Ok2xLq4wnNqlXMdMIT0s3t7UpPBYtI/VS2v5NgGUjvsOqSf+CNybH8nqZlkPHBH9qX8POln+ThWPHkvOxaRVcd461jkeV9qfhZWNJ6S/ZTG05fSON/o48uwUtl97k/ShqRfRrOBg4D3k5ocIN9ngaycrYHLgeMj4s6S5Q0pn/zHpdL/AnRurlthHfmCIp1lvgU4vrdtspSk3m/RWcCIrGdK77qNSe3es/oZxgLe3v799xVifTQivhcRewMXkWqblcwCdsx6BvTahfST7s+Vn9IcktYjfTGeHhG3ZrXetVi+99Gi7O+AFdzHusClpCac84DLJK1d42nTeCt5T8uW3QlMpO/27tJ4VyTWPO9Lpc/Ctg3af6V4ekjttQBkx20b+v+ZrmQsKYmeGBH3RsSfSP8/uWX9va8FfhgRF65A+XmOXauPS8fryOSd+SfSt+0Dkg6StLmkLSQdx1s/O28l9fC4XNL7sz7JlwO/B27v5/5vB8ZI+rSkTSV9mfRzHgBJq0s6L7tgYZSk7Un/9H19kM4nfWjPlzRa0t6kngCTI+LVfsZar7+RarSfyV7bONJJndJay3OkM/p7Zv3t16lzH9/P9vFNUjvoy6QkXs00UlPFdryVqKcB/0g6R/FMlec+AWydfU4GZ00teeR5X24HPiJp36z87wDrV9j/dtlnYXDZl0FuEfEocA3wA0m7StoG+G/S+ZUrVqTMGh4l5YETJG0k6TDSCcR6TCU1T5wlaXjJNCBn+U8Aq0naIzt2a5TvoA3HpeN1bPKOiL8A7yOdzDmDlLBvB/Yl/dTv/Qm8P6lmNI3ULWgesH/Jz+MV3f8tpPa704AHSSe/zi/ZZAlvdV16hHRy7l5St6hK5T1D6s43htRWeTHwE+CU/sS5IrL23UNIPUZmkpLqv5BOIvVusxj4AumXxFzSP04ukj5Jep8+ERFvZu3jhwMHZv+8pVfpjS/Z52zS+/dIRCzIFt9BqpVNq7HbH5J+mj9A+jzsXH3zZfvM875cXDLdTeqOdnVZUWeSapCzsv335zqETwG/JdVmf0s6gTshIl7rR5kVRcQMUm+ZL5FiPwY4uc5iPkg63s+Qel/1TuvnKT8i7iF92f+EdOy+3Md+WnZcikD9zHFmK0TSp0g13M0j4oV2x2NWNB1b87autxfpYhEnbrMV4Jq3mVkTSVqfdPJ+OKnn0pSIOKdsGwHnkCo1rwJHRcTvq5XbsWObmJl1icXASRHxe0lrAQ9K+lVElHZu+AjpQqTNSF13L8j+9snNJmZmTRQRz/bWoiPiZdKJ9fLrHvYDLo3kPmBdSVWH6ihMzXuS5PadJpv02vO1NzIrgtXWy3uVdp/qyTn/lnrATSxZNCUippRvl12TMoY0EFqpESw/Hs6cbNmzfe2zMMnbzKxTZYn6bcm6VHbB4VTghIh4qXx1pWKrlefkbWZWQSPblLOLxqYCl0fEzytsMoflL/wayVvj8FTkNm8zswoG1DFVk/UkuQiYHRHf6WOza4EjlOwAvBgRfTaZgGveZmYVNbBmuzPwSeAhSb0jQZ5CdhVuRHyfNOb6XqThiV8lXU1alZO3mVkF/T7jmYmI39QqLhvO43P1lOvkbWZWQae3KTt5m5lV0Kiad7M4eZuZVeCat5lZATXizhrN5ORtZlaBa95mZgXk5G1mVkA+YWlmVkCueZuZFZBPWJqZFZBr3mZmBeQ2bzOzAnLN28ysgJy8zcwKyM0mZmYF1OnJsdPjMzNrC9e8zcwKyG3eZmYF5Jq3mVkBueZtZlZAvjzezKyAXPM2MysgJ28zswLyCUszswJST2enbydvM7MKJCdvM7PCGTCgs1u9Ozu6Alp75EiOvP12PjdrFv80cybbf+EL7Q6pa911933sue+h7LHPQUy56NJ2h9OVVuZjrB7lntrBNe8GW7p4Mb886SSenT6dQe94B5998EEe/9WvWDB7drtD6ypLlizh1NPP5Ec/OIdhw4Zy4OFHs9v4Xdl0k43aHVrXWNmPcac3m7jm3WAL583j2enTAVi0cCELZs9mrREj2hxV95kxcxYbrj+S9UeOYNAqq7D3hA9z27RftzusrrKyH+NOr3m3NHlLettXdqVl3WLdDTfk3WPG8Mz997c7lK4z/7kFDB8+bNn8sKFDmD9/QRsj6j4r+zGWlHtqh1bXvKdWWHZVi2NoiUFrrsnBU6dy8wkn8MbLL7c7nK4T8fZlnf4zt2hW9mPc6TXvlrR5S9oC2ApYR9LHS1atDaxW5XkTgYkA+wDvb2aQDdQzcCAHT53KQ5dfzuyrr253OF1p+LAhzJs3f9n8/OcWMHTo4DZG1H1W9mPs3ibJ5qT8uy7w0ZLpfcBn+npSREyJiLERMbYoiRtgv4su4vnZs7n37LPbHUrX2mar0Tzx1ByenjOXRW++yQ0338pu43Zpd1hdZWU/xp3ebNKSmndEXANcI2nHiLi3Fftslw123pltjziC+TNmcGx24vK2U07h0ZtuanNk3WXgwIF882tf4pjjTmTJ0iUcsP8+bLbpxu0Oq6us7Me406+wVFRq2GrWzqQhpJr2KEq+OCLi07WeO0lqXaArqUmvPd/uEMwaY7X1+p15fzN0ndw5Z5fnXmx5pm91P+9rgF8DtwJLWrxvM7PcGlnzlnQxqen4uYjYuo9txgPfBVYBno+IcdXKbHXyXiMivtLifZqZ1a2nsScsLwEmAxUvU5W0LnA+MCEinpI0tGZ8jYwuh+sl7dXifZqZ1a2RJywj4i7gr1U2ORz4eUQ8lW3/XK0yW528v0hK4K9JeknSy5JeanEMZmY19fQo9yRpoqQHSqaJde7u74B3Spom6UFJR9R6QkubTSJiLUnvAjajSv9uM7N2q6cLYERMAab0Y3cDSZey7A6sDtwr6b6I+FO1J7SMpGNIte+RwP8COwD3kAI2M+sYLe4qOId0kvIV4BVJdwHbAn0m73Y0m3wAeDIiPgSMAdw/zcw6Tosv0rkG2FXSQElrANsDVYcibXVvk9cj4vXsBa8aEQ9L2rzFMZiZ1dQzoKFdBX8CjAcGS5oD/CupSyAR8f2ImC3pZmAGsBS4MCJmViuz1cl7TtYl5hfAryT9DZjb4hjMzGpqZLNJRByWY5tvA9/OW2arT1h+LHs4SdIdwDrAza2Mwcwsj54OH0GxbXfSiYg727VvM7NaOn1sE98Gzcysgk4fu9zJ28ysAte8zcwKqMFjmzSck7eZWQVuNjEzKyD1uOZtZlY4bvM2MysiN5uYmRVPz8AB7Q6hKidvM7MKfMLSzKyI3OZtZlY87m1iZlZAbjYxMysgdxU0MysgDXBvEzOzwnHN28ysgJy8zcyKSO5tYmZWOK55m5kVkJO3mVkBubdJg0x67fl2h2DWb0vO+3y7Q1gpDDjpin6X4Zq3mVkB+QpLM7MC8tgmZmYFVNhmE0l75S0kIm5sTDhmZp1BBb57/PU5ywigs0/LmpnVqcjNJqu3LAozsw5T2BOWEfFGKwMxM+soHd7mnft3gaTdJF0labqkkdmyoySNa154ZmbtoZ6e3FM75NqrpIOA64AFwBbAoGzVGsBXmxOamVn7SMo9tUPer4yvA8dGxHHA4pLl9wBjGh6VmVmbaUBP7qkd8vbz/jvgrgrLXwLWbVw4ZmadodP7eef9ypgHbFph+c7A440Lx8ysQ6iOqVZR0sWSnpM0s4/1n5A0I5vukbRtrTLzJu+LgO9Kej+pX/cwSYcA3wam5CzDzKw4pPxTbZcAE6qs/wswLiLeC3yLHHk1b7PJ6cC7SG3cqwB3k9q+z4mI7+Ysw8ysMBp5HjIi7pI0qsr6e0pm7wNG1iozV/KOiABOknQqsA2pxv5QRPwtz/PNzIpGA/Jnb0kTgYkli6ZExIq2ShwN3FRro3oHpnqF1P4N8HK9EZmZFUYdVe8sUfe7CVnSh0jJe5da2+bt572KpP8EXgAeyaYXJJ0haVD1Z5uZFVBPHVMDSHovcCGwX0T8X63t89a8JwP7Al8E7s2W7UhqWF8X+Gz9oZqZda5WXnwjaQPg58AnI+JPeZ6TN3kfChwSETeXLJslaS5wJU7eZtZtGpi8Jf0EGA8MljQH+FdS5w8i4vvAN4H1gPOzL43FETG2Wpl5k/frwJMVlj8BLMpZhplZYTS4t8lhNdYfAxxTT5l5W2suAE4pbd+WtAppXJML6tmhmVkhDFD+qQ2q3UnnZ2WLJgD/IGl6Nv/3pDG/b2lSbGZmbVPY8byBJWXzN5TN39HgWMzMOkdn5+6qN2Oo2kZjZtbNOn1gKt893sysks7O3fmTt6TDgMOADXjrZgwARMSWDY7LzKy9OrzNO+8VlicA3wf+TLqTzu3A08B7gKuaFp2ZWZtogHJP7ZC3q+BxwMSIOBF4E/hOROwJfA8Y0qzgzMzapbEjwjZe3uS9PmmYQoDXgLWyx5cBBzc6KDOztuvw7J03ec8njecN8BSwXfZ4Qzq+Wd/MrH4dnrtzn7C8A9gHmA78mHRXnY8D2wPXNCk2M7P26ZKugsf2bhsR50p6iXT/ytuAc5sUm5lZ23R6P+9czSYRsSgiXi2Z/3FETIyIMyPijeaFV0x33X0fe+57KHvscxBTLrq03eF0LR/n5tKeE+k57gJ6jjyj3aG0R4e3m1Qb2yR33+2ImNWYcIpvyZIlnHr6mfzoB+cwbNhQDjz8aHYbvyubbrJRu0PrKj7OzRcz7yKm/5KejxzX7lDaQg26yUKzVGs2mUm6U3wlytb1/h2QZ2eSdo6Iu2stK7IZM2ex4fojWX/kCAD2nvBhbpv2ayeVBvNxboFnHoa1B7c7ivbp8It0qiXv0U3Y37nA+3IsK6z5zy1g+PBhy+aHDR3CjIf8w6TRfJyt6To7d1cdmOqRRu1E0o7ATsAQSV8qWbU2VWrtpXdk/sHks5h49JGNCqlposJvlU4fWrKIfJyt2Tr989SqgakGAe/I9rdWyfKXgAP7etJyd2R+/f/6asLpKMOHDWHevPnL5uc/t4ChQ1fin55N4uNsTTegsxu9W5K8I+JO4E5Jl0REpdupdY1tthrNE0/N4ek5cxk2bAg33HwrZ/3HpHaH1XV8nK3pOryrYKuHhH1V0reBrYDVehdGxG4tjqNpBg4cyDe/9iWOOe5ElixdwgH778Nmm27c7rC6jo9z82nv49HI0bD6WvRMPJe4Zyoxc1q7w2qdDu9u0urkfTnwU9LVmscCRwILWhxD043bdSfG7bpTu8Poej7OzRU3TO6zu9lKocPbvOv6apH0DknbZjcfXhHrRcRFwJsRcWdEfBrYYQXLMjNrnh7ln9oRXp6NJK0p6VLSCcYHSaMMImmypK/Xsb83s7/PStpb0hhgZD0Bm5m1hHryT22Qd6//QboJw07A6yXLfwkcVMf+/l3SOsBJwMnAhcAJdTzfzKw1BvTkn9og7173Az4fEfex/FWXs4B6zhIdBCgiZkbEh4A9gI/V8Xwzs9bo6ck/tUHeE5ZDgOcqLF+zzv29NyJe6J2JiL9mTSdmZp2lS05YPgjsVTLfW/v+NHBvPfuT9M7eGUnvwnewN7NO1CU1768DN0raInvO5yRtBYwHxtWxv7OAeyRdRfoCOBg4rY7nm5m1RjfUvCPiLlKSHgo8A3wceAXYOSJ+m3dnEXEpcADptmoLgI9HxGX1Bm1m1nQDBuSf2iB3k0VEPAgc0t8dZmN/e/g3M+tsHV7zzpW8Ja1RbX3pXXbMzLpBp98GLW/NeyF935gBct6MwcysMLpkbJOPlM2vAowBjgH+paERmZl1gm6oeUfELRUWXy/pT8A/Ar77q5l1l25o867iAeDiRgRiZtZR2tSLJK8VbtSRNAj4HKnroJlZd5HyTzWL0gRJj0h6TNJXK6zfQNIdkqZLmiFpr0rllMrb22QBy5+wFLAusAg4Ik8ZZmaF0qArJyUNAM4jjeU0B/idpGuzbtO9vgH8LCIukLQlcCMwqlq5eZtNvlE2v5R0kc09EVFpzBMzs2JrXJv3dsBjEfF4KlZXkgb7K03eQbohO8A6wNxahdZM3pIGksbhvjEi5tUZtJlZMdXR20TSRGBiyaIp2Q3UAUYAT5esmwNsX1bEJOCXkj5PGvDvw7X2WTN5R8RiSZOB0bW2NTPrGnX0884S9ZQ+Vlf6Fii/buYw4JKIOEvSjsBlkraOiKV97TNvs8lvgW2Brr7zu5nZMo27ycIcsruPZUby9maRo4EJABFxr6TVgMFUHoobyJ+8JwNnSXoPaXjYV0pXljW8m5kVX+OGev0dsJmkjUi98w4FDi/b5ilgd+ASSaOB1ahxc/a8yftn2d/zs7+9VX5ljzu7Q6SZWb0adMIya3o+HriFlCsvjog/SjoVeCAiriXdGvKHkk4k5dSjIqLakCS5k7fbu81s5dLAKywj4kZS97/SZd8seTwL2LmeMqsmb0kXA1+MiEfqKdTMrPA6fGCqWtEdCazeikDMzDpKj/JPbVCr2aSzR2YxM2uWLhiYqmqjuZlZV+qC5D1PNV5ERLi3iZl1meIn74nAC80OxMyso3R27s6VvK/z4FNmttIpeLOJ27vNbOXUuCssm6IwvU2WnPf5dofQ9QZ87tx2h9D1vnXyT9odwkph0klXNKCUjkl/FVVN3hHR2V89ZmbN0tm5u9/3sDQz604Fb/M2M1s5OXmbmRVQh49t4uRtZlaJa95mZgXU2bnbydvMrCLXvM3MisjJ28yseFzzNjMroDbdZCEvJ28zs0pc8zYzKyInbzOz4nHN28ysgJy8zcwKyMnbzKyAnLzNzArIydvMrICcvM3MisjJ28yseFzzNjMrIA1odwRVOXmbmVXimreZWQH5NmhmZkXkmreZWfF0eLNJZ/8uMDNrm546puokTZD0iKTHJH21ynYHSgpJY2uV6Zq3mVklPY2p20oaAJwH7AHMAX4n6dqImFW23VrAF4D7c4XXkOjMzLqO6piq2g54LCIej4hFwJXAfhW2+xbwX8DreaJz8jYzq0Q9uSdJEyU9UDJNLClpBPB0yfycbNlbu5LGAOtHxPV5w3OziZlZJXWcsIyIKcCUvkqq9JS3dqMe4GzgqDqic83bzKyyhjWbzAHWL5kfCcwtmV8L2BqYJukJYAfg2lonLV3zbgLtORFtPAZefYmlP/5Ku8PpWnfdfR+nnfFdli5dwkEf+ygTjz6i3SF1lbVHjuRjl17KO4YPJ5Yu5cEpU7j/e99rd1it07iLdH4HbCZpI+AZ4FDg8N6VEfEiMHjZbqVpwMkR8UC1Ql3zboKYeRdLp57R7jC62pIlSzj19DO58PyzuOHqK7j+5lt57M9/aXdYXWXp4sX88qSTOG/LLblwhx3Y7nOfY8jo0e0Oq2WkntxTNRGxGDgeuAWYDfwsIv4o6VRJ+65ofK55N8MzD8Pag2tvZytsxsxZbLj+SNYfmc777D3hw9w27ddsuslGbY6seyycN4+F8+YBsGjhQhbMns1aI0awYPbsNkfWIg28PD4ibgRuLFv2zT62HZ+nTNe8rZDmP7eA4cOHLZsfNnQI8+cvaGNE3W3dDTfk3WPG8Mz9ubogd4mGtXk3RUtq3pKuo+TsarmIWOGfDrZyigqfJnX45cxFNWjNNTl46lRuPuEE3nj55XaH0zod/nlqVc37TOAs4C/Aa8APs2khMLOvJ5X2nfzhfY+1JFArhuHDhjBv3vxl8/OfW8DQoW6qarSegQM5eOpUHrr8cmZffXW7w2mtOvp5t0NL9hoRd0bEncCYiDgkIq7LpsOBXao8b0pEjI2IsZ/ZYdNWhGoFsc1Wo3niqTk8PWcui958kxtuvpXdxvX5UbIVtN9FF/H87Nnce/bZ7Q6l9To8ebf6hOUQSRtHxOMAWdeZIS2Ooem09/Fo5GhYfS16Jp5L3DOVmDmt3WF1lYEDB/LNr32JY447kSVLl3DA/vuw2aYbtzusrrLBzjuz7RFHMH/GDI6dPh2A2045hUdvuqnNkbWIx/NezomkjuiPZ/OjgIl9b15MccPkvhv4rWHG7boT43bdqd1hdK2n7r6bSR3e7ttcnf3aW5a8s0tAXwI2A7bIFj8cEW+0KgYzs9w6/IurZck7IpZKOisidgT+0Kr9mpmtmM5uNml1dL+UdIDcp8vMOp2Uf2qDVrd5fwlYE1gs6XVSo1JExNotjsPMrDoNaHcEVbU0eUfEWq3cn5nZCuvwBoJWXWG5RUQ8LOl9ldZHxO9bEYeZWX5O3pCaSyaSrrIs7UWnbH63FsVhZpZPh9e8W3WFZW9f7r2AG4AXgReAa7NlZmYdxgNTlfoxqa9374juhwGXAge3OA4zs+o6vObd6uS9eURsWzJ/hyT3+TazztPhvU1a3c97uqQdemckbQ/c3eIYzMxycLMJkh4inZhcBThC0lPZ/IbArFbEYGZWFzebALBPi/ZjZtYgTt5ExJOt2I+ZWcO45m1mVkAez9vMrIicvM3MisfNJmZmReTkbWZWQE7eZmaF0+n3jHHyNjOrxL1NzMyKyDVvM7Picc3bzKyIXPM2Mysen7A0MysiJ28zs+Jxm7eZWQG52cTMrIg6O3l39u8CM7N2UU/+qVZR0gRJj0h6TNJXK6xfVdJPs/X3SxpVq0wnbzOzihpzD0tJA4DzgI8AWwKHSdqybLOjgb9FxKbA2cAZtaJz8jYzq6RxNe/tgMci4vGIWARcCexXts1+wI+zx1cBu6vG4CqFafMecNIVnd0AVYGkiRExpd1xdLOiHeNJEe0OoW5FO8YNs9p6uXOOpInAxJJFU0qO2Qjg6ZJ1c4Dty4pYtk1ELJb0IrAe8Hxf+3TNu7km1t7E+snHuPl8jGuIiCkRMaolBxEAAAcFSURBVLZkKv2yq/QlUP4tnmeb5Th5m5k11xxg/ZL5kcDcvraRNBBYB/hrtUKdvM3Mmut3wGaSNpI0CDgUuLZsm2uBI7PHBwK3R1RvYytMm3dBrXzthK3nY9x8Psb9kLVhHw/cAgwALo6IP0o6FXggIq4FLgIuk/QYqcZ9aK1yVSO5m5lZB3KziZlZATl5m5kVkJN3C0nav8KVVdZAksZL2qndcRSFpFGSZlZYfmGez2p2vK9vTnRWjZN3a+1PujzWmmc84OTdTxFxTETMKl+eXeptHcDJu58k/ULSg5L+mF1lhaSFkk6T9AdJ90kaltUG9wW+Lel/JW3S3siLRdIRkmZkx/QySR/NBvCZLunW7BiPAo4FTsyO8a7tjbowBkr6cXZ8r5K0hqRpksbCss/zqZLuB3bMBll6WNJvgI+3N/SVl3ub9JOkd0XEXyWtTurPOY50Seu+EXGdpP8CXoqIf5d0CXB9RFzVxpALR9JWwM+BnSPieUnvIl199kJEhKRjgNERcZKkScDCiDizjSEXRvaF9xdgl4i4W9LFwCxgH+DkiHhAUgCHRMTPJK0GPArsBjwG/BRYIyL2acsLWIm55t1/X5D0B+A+0hVSmwGLgN52wAeBUe0JrWvsBlwVEc8DRMRfSVep3SLpIeCfga3aGF/RPR0Rd2eP/xvYpWz9EmBq9ngL4C8R8Wh2Ecl/tyhGK+Pk3Q+SxgMfBnaMiG2B6cBqwJslV0ctwRdD9Zd4+zgP5wKTI2Ib4LOk424rpvzYls+/HhFLqqy3NnDy7p91SGPwvippC2CHGtu/DKzV/LC6zm3AwZLWg9RURTr2z2TrjyzZ1se4fhtI2jF7fBjwmyrbPgxsVHLO5rCmRmZ9cvLun5tJJ3tmAN8iNZ1UcyXwz9lJNp+wzCki/gicBtyZNVF9B5gE/I+kX7P8sJnXAR/zCcu6zAaOzD7H7wIu6GvDiHidNMrgDdkJyydbE6KV8wlLM7MCcs3bzKyAnLzNzArIydvMrICcvM3MCsjJ28ysgJy8re0kzcwua++df0LSyQ3ex3hJIWlwHc+ZJmlyP/c7Ktvv2P6UY1bOydveRtIlWcIJSW9KelzSmZLWbFEIHwDOz7OhpKMkLWxyPGYdx5dtW19uBT4JrALsClwIrAkcV2ljSatExJuN2HFELGhEOWbdzDVv68sbETEvIp6OiCuAy0njkZc2Qewl6beSFgF7Zus+mg2R+7qkv2RD4w7qLVTSUEnXSHpN0pOSPl2+4/JmE0lrS7pA0rNZubMlHZKNLfMjYM2SXwqT8rw4SetJ+omkOVksf5T0qQqbDpR0jqS/ZdO3JfWUlDNI0hlZOa9I+p2kPfPEYNYfrnlbXq+RauGlzgBOIg0N+nKWtC4HvgjcBWwAfB9YFehNxpcAG5IG9HoVOJsqoy5KEnAT8E7gU8CfgM1JA1HdA5wAnA70DjeQtwllNeD32Wt4KYvnB5KeiojbSrb7RBbzjsB7gR8Cz5Iu0Yf05bEJcDgwB9gLuE7SByLiDzljMatfRHjytNxESlbXl8xvRxo/5KfZ/HjSyHIHlD3vLuBfypbtT0qoAv4ue97OJes3JI28OKlk2ROksaQB9gCWksbrrhTrUaTxu2u9pt6YB1fZ5krgwpL5aaQvC5Us+wYwJ3u8SRbbBmXl/AI4P3s8Ktvv2Ha/r566a3LN2/oyITsROJBU474G+HzZNg+Uzb8f2E7SV0qW9QCrA8OB0aRk99velRHxpKS5VeIYAzwbEbNX6FX0Qel2Xl8FDgFGkH4dDCIl7FL3RUTpAED3At+StDbwPtKX0qz0A2GZVYHbGxmvWTknb+vLXaTR494E5kblk5GvlM33AP8G/E+FbReQEl29VuQ5eZxMavL5IvAQ6dfB6cDQOsroIdWqP0A6TqVea0CMZn1y8ra+vBoRj9X5nN8DW/T1PEmzSQnvA6T2aiRtALynRpnvljS6j9r3ImBFboq7C3BdRFyWxdHbrPNC2XbbS1JJ7XsH0pfZS5Kmk75chkfEHSsQg9kKc28Ta6RTgcOzm9VuLWkLSQdm9/EkIh4hjYH+A0k7Svp7Uvt6tVrqbcD9wFRJe0raSNIekvbP1j8BrJYtGyxpjZyx/gnYXdIu2Y00JgMbVdjuPcB3JW0u6UDSLdfOzl7Pn0gnaC/JXufGksZKOlmSb8xrTeXkbQ0TEbcAewMfIrVr/5bUrvxUyWZHkW54ezvpxglXkBJwX2UuBT4C3E26X+Js4BxS+zQRcQ+pR8tPSE0zX84Z7r9n8d1EaiJ6hZSIy11OqtnfT+ppchFZ8s58itTj5L9Id5m5HvggvkmBNZlvxmBmVkCueZuZFZCTt5lZATl5m5kVkJO3mVkBOXmbmRWQk7eZWQE5eZuZFZCTt5lZAf0/m59+sqVXYc4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 绘制混淆矩阵\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import seaborn as sns\n",
    "\n",
    "cm=confusion_matrix(y_true, y_pred)\n",
    "df = pd.DataFrame(cm, columns=labels,index=labels)\n",
    "sns.heatmap(df,annot=True,cmap=\"OrRd\")\n",
    "plt.xlabel('Predict label', color='k', size=14) \n",
    "plt.ylabel('True label', color='k', size=14) \n",
    "plt.title('Confusion matrix,without normalization', color='k', size=14) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "其中参数 `cmap` 颜色模式如下\n",
    "![](https://tva1.sinaimg.cn/large/007S8ZIlly1gfpignsgybj30l20jfk0w.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "准确率 0.6666666666666666\n"
     ]
    }
   ],
   "source": [
    "# 输出准确率\n",
    "from sklearn.metrics import accuracy_score\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")  # 忽略当某个类别某有预测到时出的警告\n",
    "\n",
    "y_true = [\"cat\", \"ant\", \"cat\", \"cat\", \"ant\", \"bird\"]\n",
    "y_pred = [\"ant\", \"ant\", \"cat\", \"cat\", \"ant\", \"cat\"]\n",
    "labels=list(set(y_true))\n",
    "\n",
    "print(\"准确率\", accuracy_score(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "y_true = np.array([0, 0, 1, 1])\n",
    "y_scores = np.array([0.1, 0.4, 0.35, 0.8])\n",
    "roc_auc_score(y_true, y_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## roc_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de5xN9frA8c8zM+bGuAxSkVxzv9UkUohcQiXVQaI6nVNSCSU5VNJdInLt6nT6lUopp1yilHKIUSKXIsRI7sa4zJjL8/tjLWMbM3v2MHv2zJ7n/XrNy173Zy1772ev73d9v19RVYwxxpichAQ6AGOMMYWbJQpjjDFeWaIwxhjjlSUKY4wxXlmiMMYY45UlCmOMMV5ZoggCItJHRL4MdByBJiJVReSIiIQW4DGriYiKSFhBHdOfRGSdiLQ9i+2C9j0oIm1FJCHQcQSSJYp8JiLbROS4+4X1l4jMEJFS/jymqv6fqnb05zEKI/daX3tyWlW3q2opVU0PZFyB4iasWueyD1VtoKrf5HKcM5JjcX0PFheWKPzjelUtBTQFmgHDAxzPWQnkr+Rg+YWeF3a9TWFlicKPVPUvYAFOwgBARCJEZKyIbBeR3SIyTUSiPJbfKCKrReSwiPwuIp3d+WVE5E0R2SUiO0XkmZNFLCJyp4h8776eJiJjPeMQkc9EZIj7+kIR+VhE9orIVhEZ6LHeKBGZJSLvishh4M6s5+TG8Y67/R8iMlJEQjziWCoir4pIoohsFJH2Wbb1dg5LRWS8iBwARolITRH5WkT2i8g+Efk/ESnrrv8foCrwX/fu7dGsv3RF5BsRedrdb5KIfCkiFTzi6eeew34ReTzrHUqW844SkZfd9RNF5HvP/zegj/t/uk9ERnhs11xElonIIfe8J4lIuMdyFZH7RWQTsMmdN0FEdrjvgVUicrXH+qEi8i/3vZHkLr9IRJa4q/zsXo+e7vrd3PfTIRH5n4g09tjXNhEZJiJrgKMiEuZ5DdzY4904dovIOHfTk8c65B6rped70N22gYgsFJED7rb/yuG65vh5cGNb7vH/eZ84RWOR7vRH4ty1J4rIEhFp4LHfGSIyRUTmuTEuFZHzReQVETnovjebZbkWw0Vkvbv87ZPHySbmHD9DQUtV7S8f/4BtwLXu6yrAWmCCx/JXgDlALBAD/Bd43l3WHEgEOuAk8cpAXXfZp8B0oCRwHrACuNdddifwvfu6NbADEHe6HHAcuNDd5yrgCSAcqAFsATq5644CUoHu7rpR2ZzfO8BnbuzVgN+Auz3iSAMGAyWAnu75xPp4DmnAg0AYEAXUcq9FBFAR5wvqleyutTtdDVAgzJ3+BvgduMTd3zfAC+6y+sAR4Cr3Wox1z/3aHP5fJ7vbVwZCgSvduE4e83X3GE2AFKCeu91lQAv3nKoBG4BBHvtVYCHO+yHKnXc7UN7d5mHgLyDSXTYU5z1VBxD3eOU99lXLY9+XAnuAK9yY73CvWYTH9VsNXORx7MxrCiwD+rqvSwEtsrvO2bwHY4BdbuyR7vQVOVxXb5+HEPf/fBRQGzgINPPY9u/uNhHuflZ7LJsB7HOvfyTwNbAV6Odei2eAxVneS7+41yIWWAo84y5rCyR4xJTjZyhY/wIeQLD9uW+4I0CS+2H6CijrLhPgKFDTY/2WwFb39XRgfDb7rITz5RPlMa/3yTd6lg+pANuB1u70P4Gv3ddXANuz7Hs48Lb7ehSwxMu5hbpx1PeYdy/wjUccf+ImKXfeCqCvj+ewPadju+t0B37Kcq1zSxQjPZYPAOa7r58A3vdYFg2cIJtE4X45HAeaZLPs5DGrZDnnXjmcwyBgtse0Au1yOe+DJ48N/ArcmMN6WRPFVODpLOv8CrTxuH5/z+b9ezJRLAGeAirkcM45JYrenv9PXs7L6+fB41gHcBLscC/7KuvGVMadngG87rH8QWCDx3Qj4FCW8+7vMd0F+N193ZZTicLrZyhY/6xc0j+6q+oiEWkDvAdUAA7h/CqOBlaJyMl1BecLGJxfM3Oz2d/FOL/Qd3lsF4Jz53AaVVURmYnzYV0C3Aa867GfC0XkkMcmocB3HtNn7NNDBZxfUX94zPsD51f2STvV/fR4LL/Qx3M47dgich4wEbga55djCM6XZl785fH6GM4vY9yYMo+nqsdEZH8O+6iA86v097weR0QuAcYBcTj/92E4v0g9ZT3vh4F/uDEqUNqNAZz3iLc4PF0M3CEiD3rMC3f3m+2xs7gbGA1sFJGtwFOq+rkPx/U1xtw+D6jqNhFZjPPFPTlzJafI8lngVnc/Ge6iCjh3sQC7PY51PJvprA+ZeF6Lk+/brHz5DAUdq6PwI1X9FueXzck6g304b9AGqlrW/SujTsU3OG/UmtnsagfOr/EKHtuVVtUG2awL8D5wi4hcjPML6GOP/Wz12EdZVY1R1S6eYXs5pX04xTMXe8yrCuz0mK4sHp96d/mfPp5D1mM/785rrKqlcYpkxMv6ebELp2gQcOogcIp7srMPSCb7/5vcTAU2ArXdc/gXp58DeJyHWx8xDPgbUE5Vy+J88Z3cJqf3SHZ2AM9m+f+OVtX3szt2Vqq6SVV74xQTvgjMEpGS3rbJY4y5fR4QkS44dxlfAS95bHsbcCNwLVAG584Dzry2eXGRx+uT79usfPkMBR1LFP73CtBBRJqqagZOWfZ499cyIlJZRDq5674J3CUi7UUkxF1WV1V3AV8CL4tIaXdZTfeO5Qyq+hOwF3gDWKCqJ3/9rAAOu5WEUW7FaEMRudyXE1HnsdMPgWdFJMZNREM4dccCzpfKQBEpISK3AvWAuXk9B1cMTjHeIRGpjFM+72k3Thnx2ZgFXC8iV4pTufwUOXzJuP9vbwHj3IrMULcCN8KH48QAh4EjIlIXuM+H9dNw/v/CROQJnDuKk94AnhaR2uJoLCInE1zW6/E60F9ErnDXLSkiXUUkxoe4EZHbRaSie/4n30PpbmwZ5HztPwfOF5FBbmV1jIhckXWl3D4P4jx48CbO3dUdOP9fJ7+QY3B+eOzHuSt5zpdzysX9IlJFRGJxEvoH2axzTp+hosoShZ+p6l6cCuDH3VnDgM3AcnGeLFqEUzGJqq4A7gLG4/yK/JZTv9774RQbrMcpfpkFXODl0O/j/Np6zyOWdOB6nKewtuL8onsD5xeZrx7EKVfeAnzv7v8tj+U/4FQ87sMpGrhFVU8W6eT1HJ7CqZBNBL4APsmy/HlgpDhP9DySh3NAVde55zIT5+4iCafiNyWHTR7BqUReiVNm/iK+fX4ewfn1m4TzpZjdl4+nBcA8nIcE/sC5k/EsEhmHk6y/xElAb+JUooNTx/Rv93r8TVXjceqoJuFc781k8ySbF52BdSJyBJiAU++SrKrHcP5vl7rHauG5kaom4TyEcD1Okdwm4JocjpHj5wF4DfhMVee676G7gTfcxPiOe3124ryflufhvHLyHs513eL+PZN1hXz6DBU5J5+MMeacicidwD9U9apAx5JX4jSKPIRTRLQ10PGYgiUi23Deu4sCHUthZHcUptgSketFJNotdx+Lc8ewLbBRGVP4WKIwxdmNOBWWf+IUl/VSu8U25gxW9GSMMcYru6MwxhjjVZFrcFehQgWtVq1aoMMwxpgiZdWqVftUteLZbFvkEkW1atWIj48PdBjGGFOkiMgfua+VPSt6MsYY45UlCmOMMV5ZojDGGOOVJQpjjDFeWaIwxhjjlSUKY4wxXvktUYjIWyKyR0R+yWG5iMhEEdksImtE5FJ/xWKMMebs+fOOYgZON8U5uQ6nf53awD04A7wYY4zJZydOpJ/T9n5rcKeqS0SkmpdVbgTecTthWy4iZUXkAneAG2PMSZ90ha3ZjZBrTO6G/rcDP/3pbdiX3AWyjqIypw/IksDpYy9nEpF7RCReROL37t1bIMEZU2hYkjDnoOH5e/huS9Vz2kcgu/DIbtjJbLuyVdXXcEa7Ii4uzrq7NcXTw/bWN7lbv34vP/64i9tvbwxAP1XavJBI9epnDNjns0AmigROH8y8CtkPZm6MMSYXx46l8swzS3jppf8RGiq0aFGFWrViERGqVSt7TvsOZKKYAzwgIjOBK4BEq58wxpi8mzdvE/ffP5etWw8BcPfdl1G+fFQuW/nOb4lCRN4H2gIVRCQBeBIoAaCq04C5QBecgdWPAXf5KxZjjAlGO3ceZtCgBcyatR6Axo0rMW1aV1q2vCiXLfPGn0899c5luQL3++v4xhgT7O6/fy6fffYr0dElGD26LQ891IKwsPx/RqnIjUdhjDHFWVpaRmYyePHFaylRIpSXX+5I1apl/HZMSxTG+IO1fTD5LDExmZEjv+a33w4wf34fRIQ6dSrw0Ue3+v3YliiM8Yf8ThLVu+Tv/kyRoap89NF6Bg2az65dRwgNFVav/otmzc6tEV1eWKIwxp+s7YM5B7//foAHHpjH/PmbAWjZsgrTpnWjceNKBRqHJQpjjCmExo79H48/vpjk5DTKlo3kxRev5R//uJSQkOzaKvuXJQpjjCmEjh1LJTk5jb59GzN2bEfOO69kwGKxRGGMMYXA3r1H+fXX/Vx1ldMv07BhrWjbthqtW18c4Mhs4CJjjAmojAzljTd+pE6dSfTo8QEHDhwHICIirFAkCbA7CmOMCZhfftlD//6fs3Sp05F2hw41OHYsldjY/Ot+Iz9YojDGmAJ29OgJRo/+lnHjlpOWlkGlSiV55ZXO9OzZAJGCr6zOjSUKY3JjjedMPrvllo+YP38zIjBgQBzPPtuesmUjAx1WjixRGJObs00S1kjO5GDYsFbs3n2EqVO7csUVVQIdTq4sURjjK2s8Z85CWloGr776A9u2HWLChOsAaNu2GvHx9wSkTcTZsERhjDF+smLFTu6993NWr/4LgHvuuYwGDc4DKDJJAuzxWGOMyXeHDiUzYMAXtGjxBqtX/8XFF5fhv//tnZkkihq7ozDGmHw0c+YvDBo0n927jxIWFsLDD7fk8cdbU7JkeKBDO2uWKIwxJh99+eXv7N59lFatLmLq1K40alSwHfj5gyUKY4w5BykpaezcmUSNGuUAGDOmA1dfXZU77mhapOohvLE6CmOMOUtff72Vxo2n0bXre5w4kQ5AhQrR3HVXs6BJEmB3FMY4rFGdyYPdu4/wyCMLeffdNQDUrVuBhITDmXcVwcYShTGQe5KwxnMGpwO/119fxWOPfcWhQ8lERoYxcuTVDB3aivDw0ECH5zeWKIzxZI3qjBc33fQBc+b8CkCnTjWZPLkLNWvGBjgq/7M6CmOM8VGPHnU5//xSfPDBLcyb16dYJAmwOwpjjMnRnDm/kpBwmAEDLgegX78m9OhRj5iYiABHVrAsURhjTBbbtycycOA8PvvsVyIiQuncuRY1apRDRIpdkgBLFMYYkyk1NZ2JE3/gySe/4ejRVGJiwnnmmXZcfHGZQIcWUJYojDEGWL48gXvv/Zw1a3YDcOut9Rk/vhOVK5cOcGSBZ4nCBB9rE2HOwuOPL2bNmt1Ur16WSZO60KVL7UCHVGhYojDBxwYaMj5QVZKSTlC6tFPnMGnSdbzzzs+MGNGa6OgSAY6ucLFEYYKXtYkwOfj1130MGDAXEVi4sC8iQp06FXj22faBDq1QskRhjCk2kpPTeP7573jhhaWcOJFO+fJRbNt2iOrVg7PrjfxiicIYUywsXPg7AwbMZfPmAwD8/e9NGTOmA+XLRwc4ssLPry2zRaSziPwqIptF5LFsllcVkcUi8pOIrBERKyQ2xuQrVeXvf/+Mjh3fZfPmA9SvX5ElS+7kzTdvtCThI7/dUYhIKDAZ6AAkACtFZI6qrvdYbSTwoapOFZH6wFygmr9iMsYUPyJCtWpliYoK44kn2jBkSMug7sDPH/xZ9NQc2KyqWwBEZCZwI+CZKBQ4+ZByGeBPP8ZjjCkmVq/+i127krjuOucR12HDWtG3b2OrizhL/ix6qgzs8JhOcOd5GgXcLiIJOHcTD2a3IxG5R0TiRSR+7969/ojVGBMEkpJSGDJkAZdd9hp33PEpBw4cByAiIsySxDnw5x1FdsM7ZX1esTcwQ1VfFpGWwH9EpKGqZpy2keprwGsAcXFx9sxjsLEGcuYcqSqffrqRgQPnk5BwmJAQ4bbbGlGihHWQnR/8mSgSgIs8pqtwZtHS3UBnAFVdJiKRQAVgjx/jMoWNP5KENZ4rNv744xAPPDCPzz//DYC4uAuZPr0bl156QYAjCx7+TBQrgdoiUh3YCfQCbsuyznagPTBDROoBkYCVLRVX1kDO5JGqcvPNH7Jq1S5Kl47guefa0b9/HKGhdieRn/yWKFQ1TUQeABYAocBbqrpOREYD8ao6B3gYeF1EBuMUS92pqvZtYYzxKiNDCQkRRISxYzsybVo848d34oILYgIdWlCSova9HBcXp/Hx8YEOw+Snl93qLLujMLnYv/8Yjz22CIDXX78hwNEULSKySlXjzmZbuz8zxhR6qsq//72aunUn88YbP/HOO2tISDgc6LCKDevCwxhTqG3YsJf77vuCb7/9A4C2basxdWpXqlSxcSIKiiUKY0yhpKo88cRiXnxxKampGVSoEM3LL3ekb9/GiGT39L3xF0sUpuBYewmTByLCzp1JpKZm8M9/XsoLL1xLbGxUoMMqlixRmILjLUlYuwcD/PlnEvv2HaNx40oAjBnTgbvvbkarVlUDHFnxZonCFDx7uslkkZ6ewdSp8YwY8TWVK8ewenV/wsNDqVAhmgoVLEkEmiUKY0xA/fjjLu6993Pi452OG1q3vpjDh1OoUMG6AC8sfEoUIhIOVFXVzX6OxxhTTBw+nMLjj3/NpEkrychQqlQpzcSJneneva5VVhcyuSYKEekKjAPCgeoi0hR4UlVv8ndwxpjgpKq0bv02P/+8m9BQYciQFowa1ZaYmIhAh2ay4UuDu9HAFcAhAFVdDdTyZ1DGmOAmIgwe3ILmzSsTH38PL7/cyZJEIeZL0VOqqh7KcitotZHGGJ+dOJHOuHHLCA0Vhg5tBUC/fk24/fbG1oFfEeBLotggIn8DQtyeYB8Clvs3LGNMsPjuuz/o3/8L1q/fS0REKP36NaFSpVKICKGhVhdRFPiSKB4AngAygE9weoMd7s+gTBFmjeqMa9++Yzz66ELefns1ALVrxzJlSlcqVSoV4MhMXvmSKDqp6jBg2MkZItIDJ2kYc7rckoQ1rAt6qsqMGasZOnQh+/cfJzw8lOHDr+Kxx64iMtKeyC+KfPlfG8mZSWFENvOMOcUa1RVr7767lv37j9OuXXWmTOlCnToVAh2SOQc5JgoR6YQzTGllERnnsag0TjGUMcYAcOxYKomJyVxwQQwiwpQpXVi58k/69GlkbSKCgLc7ij3AL0AysM5jfhLwmD+DMsYUHfPmbeL+++dSo0Y5Fi7si4hQp04Fu4sIIjkmClX9CfhJRP5PVZMLMCZjTBGwc+dhBg1awKxZ6wGIiYlg//7j1vVGEPKljqKyiDwL1AciT85U1Uv8FpUxptBKT89g8uSVjBz5NUlJJyhZsgSjR1/DwIFXEBZmbSKCkS+JYgbwDDAWuA64C6ujMKZYyshQ2rSZwdKlOwDo3r0uEyZ0pmrVMgGOzPiTL+k/WlUXAKjq76o6ErjGv2EZYwqjkBChY8eaXHRRaT77rBezZ/e0JFEM+HJHkSLOYwu/i0h/YCdwnn/DMsYUBqrKhx+uIywshJtvrg/AsGGtGDKkJaVKhQc4OlNQfEkUg4FSwEDgWaAM8Hd/BmWMCbzffz/AgAFz+fLL36lYMZp27apTrlwUERFhRFj/fcVKrolCVX9wXyYBfQFEpIo/gzLGBE5KShovvfQ/nn32O5KT0yhXLpJnn21HmTKRuW9sgpLXRCEilwOVge9VdZ+INMDpyqMdYMnCmCDzzTfbuO++L9i4cR8Affs2ZuzYjpx3XskAR2YCKcfKbBF5Hvg/oA8wX0RGAIuBnwF7NNaYIJOensGAAU6SqFOnPF9/3Y933rnJkoTxekdxI9BEVY+LSCzwpzv9a8GEZozxt4wMJTk5jejoEoSGhjB1aleWLPmDRx9tRUSEdeBnHN7eCcmqehxAVQ+IyEZLEsYEj7Vrd9O//xfUrVueN9+8EYA2barRpk21wAZmCh1viaKGiJzsIVaAah7TqGoPv0ZmjPGLo0dPMHr0t4wbt5y0tAy2bj3IwYPHKVcuKtChmULKW6K4Ocv0JH8GYozxv//+91ceeGAe27cnIgIDBsTx7LPtKVvWnmgyOfPWKeBXBRmIMcZ/0tIy6NlzFp98sgGApk3PZ/r0bjRvXjnAkZmiwGqrjCkGwsJCKFMmglKlwnn66Wt44IHm1oGf8Zlf3yki0llEfhWRzSKS7RgWIvI3EVkvIutE5D1/xmNMcfLDDwn88ENC5vRLL3Vgw4b7GTSohSUJkyc+31GISISqpuRh/VBgMtABSABWisgcVV3vsU5tYDjQSlUPioj1IWXMOTp0KJnhwxcxffoq6tatwOrV/QkPD6V8eRsnwpydXH9WiEhzEVkLbHKnm4jIqz7suzmwWVW3qOoJYCZO2wxP/wQmq+pBAFXdk6fojTGZVJX33ltL3bqTmDZtFaGhIdxwQx3S021UAHNufLmjmAh0Az4FUNWfRcSXbsYrAzs8phOAK7KscwmAiCwFQoFRqjrfh30bYzxs2rSfAQPmsmjRFgBatbqIadO60bCh3aSbc+dLoghR1T+yDJCe7sN22Y2ortkcvzbQFqfvqO9EpKGqHjptRyL3APcAVK1a1YdDG1N8pKam067dOyQkHCY2NooxY67lrruaERKS3UfQmLzzJVHsEJHmgLr1Dg8Cv/mwXQJwkcd0FZxuQLKus1xVU4GtIvIrTuJY6bmSqr4GvAYQFxeXNdkYUyypKiJCiRKhPPtsOxYv3saYMddSsaL1zWTyly+J4j6c4qeqwG5gkTsvNyuB2iJSHWewo17AbVnW+RToDcwQkQo4RVFbfAs9yH3SFbbODXQUphDavfsIjzyykEsuieXxx9sA0K9fE/r1axLgyEyw8iVRpKlqr7zuWFXTROQBYAFO/cNbqrpOREYD8ao6x13WUUTW4xRnDVXV/Xk9VlAqykmiepdARxCUMjKU119fxWOPfcWhQ8mULRvJoEEtiImxUYSMf/mSKFa6RUIfAJ+oapKvO1fVucDcLPOe8HitwBD3z2TnYStpM/Dzz3/Rv/8XLF/utIvo3LkWkyd3sSRhCoQvI9zVFJErcYqOnhKR1cBMVZ3p9+iMKeZSU9MZPvwrXnllOenpygUXlGLChM7cckt9sjxgYozf+NQ8U1X/p6oDgUuBwzgDGhlj/CwsLISffvqLjAzlwQebs2HD/dx6awNLEqZA5XpHISKlcBrK9QLqAZ8BV/o5LmOKre3bE0lPz6B69XKICNOmdSUxMYW4uAsDHZoppnypo/gF+C8wRlW/83M8xhRbqanpTJjwA08++Q0tW1Zh4cK+iAi1a5cPdGimmPMlUdRQVesDwBg/WrZsB/37f8GaNbsBiI2N4tixVEqWDA9wZMZ4SRQi8rKqPgx8LCJnPHpjI9wZc+4OHjzOY48t4rXXfgSgevWyTJ7cheuuqx3gyIw5xdsdxQfuvzaynT9Zw7piKyUljaZNp7N9eyIlSoQwdOiVjBjRmujoEoEOzZjTeBvhboX7sp6qnpYs3IZ0NgJefvCWJKzhWlCLiAjj7rub8dVXW5k6tSv161cMdEjGZEucNm9eVhD5UVUvzTLvJ1Vt5tfIchAXF6fx8fGBOLR/vOw+5mgN64JecnIazz//HXXqVOC22xoBzhCloaFij7savxORVaoadzbbequj6InzSGx1EfnEY1EMcCj7rYwx2Vm48HcGDJjL5s0HOO+8ktx0U12iokrYSHOmSPBWR7EC2I/T6+tkj/lJwE/+DMqYYPHXX0cYMmQB77//CwANGlRk2rRuREVZPYQpOrzVUWwFtuL0FmuMyYP09AymT1/Fv/71FYmJKURFhfHkk20YPLgl4eGhgQ7PmDzxVvT0raq2EZGDnD7gkOD05xfr9+iMKaLS05VXX11BYmIKXbrUZtKk66hevVygwzLmrHgrejo53GmFggjEmKIuKSmF9HSlbNlIwsNDef3169m9+wg9etSzympTpHkrejrZGvsi4E9VPSEiVwGNgXdxOgc0vrC2EkFNVZk9eyMDB86jU6eavPnmjQBcdZUN22uCgy+PXHyKMwxqTeAdnI4B3/NrVMEmtyRh7SWKrG3bDnHDDTO5+eYP2bkziV9+2UtyclqgwzImX/nS11OGqqaKSA/gFVWdKCL21NPZsLYSQSM1NZ1x45bx1FPfcvx4GqVLR/Dcc+3o3z+O0FB75NUEF5+GQhWRW4G+QHd3nj3bZ4qtY8dSadHiDdau3QNAr14NGTeuIxdcEBPgyIzxD18Sxd+BATjdjG8RkerA+/4Ny5jCKzq6BHFxF3LsWCpTpnSlY8eagQ7JGL/yZSjUX0RkIFBLROoCm1X1Wf+HZkzhoKq8887P1KwZm1lBPX58J8LDQ63hnCkWfBnh7mrgP8BOnDYU54tIX1Vd6u/gjAm0DRv2ct99X/Dtt39Qr14FVq/uT3h4KGXKRAY6NGMKjC9FT+OBLqq6HkBE6uEkjrPqXMqYouD48VSeffY7xoxZSmpqBhUrRjN8+FWUKGEV1ab48SVRhJ9MEgCqukFEbNgtE7Tmz9/M/ffPZcuWgwD885+X8sIL1xIbGxXgyIwJDF8SxY8iMh3nLgKgD9YpoAlSR46coG/f2ezbd4yGDc9j2rSutGplDedM8eZLougPDAQexamjWAK86s+gjClI6ekZZGQoJUqEUqpUOBMmdCYh4TCDB7egRAnrwM8Yr4lCRBoBNYHZqjqmYEIypuCsWvUn9977OTfeWIfHH28DkDmokDHGkWPNnIj8C6f7jj7AQhH5e4FFZYyfHT6cwkMPzaN58zdYtWoX//nPGlJT0wMdljGFkrc7ij5AY1U9KiIVgbnAWwUTljH+oarMmrWehx6az65dRwgNFYYMacFTT11jxUzG5MBbokhR1aMAqrpXROy5QFOkJSWl0LPnLObN2wzAFVdUZtq0bjRten6AI/QfBMAAAB6eSURBVDOmcPOWKGp4jJUtQE3PsbNVtYdfIzMmn5UqFU5KSjplykTwwgvXcs89lxESYuNEGJMbb4ni5izTk/wZiDH+sGTJH1xwQSlq1y6PiPDWWzcQGRlGpUqlAh2aMUWGt4GLvirIQIzJT/v2HePRRxfy9turad++OgsX9kVEuPjisoEOzZgix5d2FMYXNopdoZCRocyYsZqhQxdy4MBxwsNDufrqqqSnK2FhVsxkzNnwawW1iHQWkV9FZLOIPOZlvVtEREWk6PYfZaPYBdy6dXto23YGd989hwMHjtO+fXXWrr2PJ59sS1iYPYthzNny+Y5CRCJUNSUP64cCk4EOQAKwUkTmePYb5a4Xg9Py+wdf912o2Sh2AZGYmEyLFm9y5MgJzjuvJOPGdeS22xohYncRxpyrXH9miUhzEVkLbHKnm4iIL114NMcZu2KLqp4AZgI3ZrPe08AYINn3sI1xqDqJuUyZSIYNa0X//pexceP99OnT2JKEMfnEl/vxiUA3YD+Aqv4MXOPDdpWBHR7TCe68TCLSDLhIVT/3tiMRuUdE4kUkfu/evT4c2gS7nTsPc8stH/Luu2sy540YcTVTp3ajXDnr5dWY/ORLoghR1T+yzPOlr4Psfs5llsu4DfjGAw/ntiNVfU1V41Q1rmLFij4c2gSrtLQMJkxYTt26k/n44w08+eQ3pKdnANgdhDF+4ksdxQ4RaQ6oW+/wIPCbD9slABd5TFcB/vSYjgEaAt+4H/DzgTkicoOqxvsSvCleVq7cSf/+X/Djj7sA6N69LhMndiY01CqqjfEnXxLFfTjFT1WB3cAid15uVgK1RaQ6zjCqvYDbTi5U1USgwslpEfkGeMSShMnq6NETDBu2iClTVqIKVauW4dVXr+OGG+oEOjRjioVcE4Wq7sH5ks8TVU0TkQeABUAo8JaqrhOR0UC8qs7Jc7SmWAoLC2HRoi2EhAhDhrTkySfbULKkDbJoTEHJNVGIyOt41C2cpKr35Latqs7F6XXWc94TOazbNrf9meLj998PULZsJOXLRxMREcZ//nMTkZFhNGpUKdChGVPs+FK4uwj4yv1bCpwH+Nyewpi8SElJ45lnltCw4VSGDVuUOf/yyytbkjAmQHwpevrAc1pE/gMs9FtEptj65ptt3HffF2zcuA9wnnBKT8+wympjAuxs+nqqDlyc34GY4mvPnqMMHbqQd975GYA6dcozdWpXrrmmeoAjM8aAb3UUBzlVRxECHABy7LfJmLzYt+8Y9epN5sCB40REhDJixNU8+mgrIiKsv0pjCguvn0ZxGjg0wXm8FSBDT/aZYEw+qFAhmhtvrENCwmGmTOlKrVqxgQ7JGJOF10Shqiois1X1soIKyAS3o0dPMHr0t3TtegmtWzslmFOmdCUiItRaVhtTSPlSS7hCRC71eyQm6P33v79Sv/4Uxoz5HwMGfEFGhnNzGhkZZknCmEIsxzsKEQlT1TTgKuCfIvI7cBSnDydV1eKZPGyAojzbsSORhx6az+zZGwFo1ux8pk/vZuNVG1NEeCt6WgFcCnQvoFiKBm9JwgYnOk1aWgYTJ/7AE08s5ujRVEqVCueZZ67h/vub20BCxhQh3hKFAKjq7wUUS9FiAxTl6vDhFJ5//nuOHk3l5pvr8cornalSpXSgwzLG5JG3RFFRRIbktFBVx/khHlPEHTqUTFRUGBERYcTGRjF9ejciIkLp2vWSQIdmjDlL3u7/Q4FSON2BZ/dnTCZV5b331lKnziTGjFmaOb9Hj3qWJIwp4rzdUexS1dEFFokpsn77bT8DBnzBV19tBWDJku2oqj3JZEyQyLWOwpicJCen8eKL3/Pcc99z4kQ6sbFRvPRSB+68s6klCWOCiLdE0b7AojBFzl9/HaF167fZtOkAAHfe2ZSXXupAhQrRAY7MGJPfckwUqnqgIAMpVKytRK4qVSrJRReVISwshKlTu9KmTbVAh2SM8RPreS07uSWJYtheIiNDef31VVxzTXUuuaQ8IsJ77/WgXLkowsNDAx2eMcaPLFF4Y20lAPj557/o3/8Lli9PoH376ixc2BcRoVKlUoEOzRhTACxRmBwdOXKCUaO+4ZVXlpOerlx4YQz9+8cFOixjTAGzRGGy9emnG3nwwXkkJBwmJER48MHmPPNMO0qXjgh0aMaYAmaJwpxh587D9Oo1i5SUdC677AKmTetGXNyFgQ7LGBMgligMAKmp6YSFhSAiVK5cmmefbUd4eCgDBlxuY1YbU8zZN4Dhf//bwWWXvca7767JnPfww1fy4INXWJIwxliiKM4OHDjOvff+l1at3mLt2j1MmRKPjXRrjMkqeIqerJGcz1SVd99dw8MPf8nevccoUSKERx9txYgRV1vXG8aYMwRPosjvJBGkjep27z5C794fs3jxNgDatLmYqVO7Uq9excAGZowptIInUZxkjeS8Kls2kl27jlChQjRjx3agX78mdhdhjPEq+BKFOcPChb9z6aUXUL58NBERYXz00a1ccEEpype3DvyMMbmzyuwgtmtXEr17f0zHju8ybNiizPkNG55nScIY4zO7owhC6ekZTJ++iuHDv+Lw4RSiosKoU6e8DSZkjDkrliiCzI8/7qJ//89ZufJPALp2rc2kSV2oVq1sgCMzxhRVliiCyLZth2je/HXS05XKlWOYOPE6brqprt1FGGPOiV8ThYh0BiYAocAbqvpCluVDgH8AacBe4O+q+oc/Ywpm1aqV5a67mhITE8FTT7UlJsY68DPGnDu/VWaLSCgwGbgOqA/0FpH6WVb7CYhT1cbALGCMv+IJRtu2HeL669/n22+3Zc577bXrGTeukyUJY0y+8ecdRXNgs6puARCRmcCNwPqTK6jqYo/1lwO3+zGeoJGams64cct46qlvOX48jX37jrFs2d0AVsxkjMl3/kwUlYEdHtMJwBVe1r8bmJfdAhG5B7gHoGrVqvkVX5H0/ffb6d//c9at2wtAr14NGTeuY4CjMsYEM38miux+2mbbbFpEbgfigDbZLVfV14DXAOLi4opl0+uDB48zdOhC3nzzJwBq1izHlCld6dixZoAjM8YEO38migTgIo/pKsCfWVcSkWuBEUAbVU3xYzxFWkaG8tlnv1KiRAiPPXYVw4dfRVRUiUCHZYwpBvyZKFYCtUWkOrAT6AXc5rmCiDQDpgOdVXWPH2MpkjZu3Ef16mWJiAijfPlo/u//elC1ahnq1q0Q6NCMMcWI3556UtU04AFgAbAB+FBV14nIaBG5wV3tJaAU8JGIrBaROf6Kpyg5diyVESO+onHjqYwZszRzfseONS1JGGMKnF/bUajqXGBulnlPeLy+1p/HL4rmz9/MgAFfsHXrIQD27TsW4IiMMcWdtcwuJP78M4lBg+bz0UfO08ONGp3HtGnduPLKi3LZ0hhj/MsSRSHw22/7iYt7jaSkE0RHl2DUqDYMGtSCEiVCAx2aMcZYoigMateO5fLLK1OyZAleffU6Lr7YOvAzxhQeligC4PDhFJ54YjEDBlzOJZeUR0SYM6cXJUuGBzo0Y4w5gyWKAqSqzJq1nocems+uXUfYuHEf8+c7vZZYkjDGFFaWKArIli0HeeCBucybtxmAFi2q8OKL9tCXMabws0ThZydOpDN27P94+uklJCenUbZsJC+80J5//vMyQkKsAz9jTOFnicLPduxIZPTob0lJSadPn0a8/HJHKlUqFeiwjDHGZ5Yo/ODgweOULRuJiFCzZiwTJnSmVq1Y2revEejQjDEmz4peoti9Cl4unEU2GRnKjBmrGTp0Ia+80om+fZsAcO+9cQGOzBhjzp7f+noKiOpdAnbodev20LbtDO6+ew4HDhzPrLQ2xpiirujdUQA8XHiGpDh2LJWnn/6WsWOXkZaWwXnnlWT8+E707t0w0KEZY0y+KJqJopD47bf9dOr0Ltu2HUIE+ve/jOeea0+5clGBDs0YY/KNJYpzcPHFZYiMDKNJk0pMm9aNFi2qBDokU4ikpqaSkJBAcnJyoEMxxUhkZCRVqlShRIn8G9jMEkUepKVlMG1aPL17N6R8+WgiIsKYP78PlSuXJiwsuKp7zLlLSEggJiaGatWqIVI4H8AwwUVV2b9/PwkJCVSvXj3f9mvfbj5asWInzZu/zoMPzmPYsEWZ8y++uKwlCZOt5ORkypcvb0nCFBgRoXz58vl+F2t3FLlITExmxIivmTJlJapQtWoZbryxTqDDMkWEJQlT0PzxnrNEkQNV5YMP1jF48AL++usIYWEhDBnSgieeaGMd+BljihUrM8nBzz/vpnfvj/nrryNceeVF/PjjPbz4YgdLEqZICQ0NpWnTpjRs2JDrr7+eQ4cOZS5bt24d7dq145JLLqF27do8/fTTqJ569HzevHnExcVRr1496tatyyOPPBKIU/Dqp59+4h//+Eegw/Dq+eefp1atWtSpU4cFCxZku87VV19N06ZNadq0KRdeeCHdu3cH4ODBg9x00000btyY5s2b88svvwBw4sQJWrduTVpaWsGchKoWqb/LqqD+kpaWftr04MHz9fXXV2l6eobfjmmC1/r16wMdgpYsWTLzdb9+/fSZZ55RVdVjx45pjRo1dMGCBaqqevToUe3cubNOmjRJVVXXrl2rNWrU0A0bNqiqampqqk6ePDlfY0tNTT3nfdxyyy26evXqAj1mXqxbt04bN26sycnJumXLFq1Ro4ampaV53aZHjx7673//W1VVH3nkER01apSqqm7YsEHbtWuXud6oUaP03XffzXYf2b33gHg9y+9dK3pyLV68lQED5jJ9ejdat74YgHHjOgU4KhM0/NXtTB4an7Zs2ZI1a9YA8N5779GqVSs6duwIQHR0NJMmTaJt27bcf//9jBkzhhEjRlC3bl0AwsLCGDBgwBn7PHLkCA8++CDx8fGICE8++SQ333wzpUqV4siRIwDMmjWLzz//nBkzZnDnnXcSGxvLTz/9RNOmTZk9ezarV6+mbFlnVMdatWqxdOlSQkJC6N+/P9u3bwfglVdeoVWrVqcdOykpiTVr1tCkidNVzooVKxg0aBDHjx8nKiqKt99+mzp16jBjxgy++OILkpOTOXr0KF9//TUvvfQSH374ISkpKdx000089dRTAHTv3p0dO3aQnJzMQw89xD333OPz9c3OZ599Rq9evYiIiKB69erUqlWLFStW0LJly2zXT0pK4uuvv+btt98GYP369QwfPhyAunXrsm3bNnbv3k2lSpXo3r07w4cPp0+fPucUoy+KfaLYs+coQ4cu5J13fgZg3LhlmYnCmGCRnp7OV199xd133w04xU6XXXbZaevUrFmTI0eOcPjwYX755RcefvjhXPf79NNPU6ZMGdauXQs4RSW5+e2331i0aBGhoaFkZGQwe/Zs7rrrLn744QeqVatGpUqVuO222xg8eDBXXXUV27dvp1OnTmzYsOG0/cTHx9Ow4akeEOrWrcuSJUsICwtj0aJF/Otf/+Ljjz8GYNmyZaxZs4bY2Fi+/PJLNm3axIoVK1BVbrjhBpYsWULr1q156623iI2N5fjx41x++eXcfPPNlC9f/rTjDh48mMWLF59xXr169eKxxx47bd7OnTtp0aJF5nSVKlXYuXNnjtdm9uzZtG/fntKlSwPQpEkTPvnkE6666ipWrFjBH3/8QUJCApUqVaJhw4asXLky1+udH4ptosjIUN5880eGDVvEwYPJRESEMnJka4YOvTLQoZlgFKBuZ44fP07Tpk3Ztm0bl112GR06dACcIuecno7Jy1MzixYtYubMmZnT5cqVy3WbW2+9ldDQUAB69uzJ6NGjueuuu5g5cyY9e/bM3O/69esztzl8+DBJSUnExMRkztu1axcVK1bMnE5MTOSOO+5g06ZNiAipqamZyzp06EBsbCwAX375JV9++SXNmjUDnLuiTZs20bp1ayZOnMjs2bMB2LFjB5s2bTojUYwfP963iwOn1fmc5O36vv/++6fVuTz22GM89NBDNG3alEaNGtGsWTPCwpyv7dDQUMLDw8+4Lv5QLBPF1q0Huf322fzvfzsA6NixJpMnd6FWrdgAR2ZM/oqKimL16tUkJibSrVs3Jk+ezMCBA2nQoAFLliw5bd0tW7ZQqlQpYmJiaNCgAatWrcos1slJTgnHc17WZ/pLliyZ+bply5Zs3ryZvXv38umnnzJy5EgAMjIyWLZsGVFROXeHExUVddq+H3/8ca655hpmz57Ntm3baNu2bbbHVFWGDx/Ovffee9r+vvnmGxYtWsSyZcuIjo6mbdu22bZHyMsdRZUqVdixY0fmdEJCAhdeeGG257N//35WrFiRmagASpcunVkMpapUr179tIZ0KSkpREZGZru//FQsn3oqXTqC337bz/nnl2LmzJuZP7+PJQkT1MqUKcPEiRMZO3Ysqamp9OnTh++//55Fi5zGo8ePH2fgwIE8+uijAAwdOpTnnnuO3377DXC+uMeNG3fGfjt27MikSZMyp08WPVWqVIkNGzZkFi3lRES46aabGDJkCPXq1cv89Z51v6tXrz5j23r16rF586lemhMTE6lcuTIAM2bMyPGYnTp14q233sqsQ9m5cyd79uwhMTGRcuXKER0dzcaNG1m+fHm2248fP57Vq1ef8Zc1SQDccMMNzJw5k5SUFLZu3cqmTZto3rx5tvv96KOP6Nat22lf/IcOHeLEiRMAvPHGG7Ru3TqzWGr//v1UrFgxX7vqyEmxSRQLFmwmJcV5lKx8+WjmzOnFxo3307NnQ2sUZYqFZs2a0aRJE2bOnElUVBSfffYZzzzzDHXq1KFRo0ZcfvnlPPDAAwA0btyYV155hd69e1OvXj0aNmzIrl27ztjnyJEjOXjwIA0bNqRJkyaZv7RfeOEFunXrRrt27bjgggu8xtWzZ0/efffdzGIngIkTJxIfH0/jxo2pX78+06ZNO2O7unXrkpiYSFJSEgCPPvoow4cPp1WrVqSnp+d4vI4dO3LbbbfRsmVLGjVqxC233EJSUhKdO3cmLS2Nxo0b8/jjj59Wt3C2GjRowN/+9jfq169P586dmTx5cmaxW5cuXfjzzz8z1505cya9e/c+bfsNGzbQoEED6taty7x585gwYULmssWLF9OlS8EMrSDZlaEVZnEXicbv8D3mHTsSGThwPp9+upGnn76GkSNb+zE6Y07ZsGED9erVC3QYQW38+PHExMQU+rYU/tCjRw+ef/556tQ5s6eI7N57IrJKVc9qFLWgvaNIS8tg3Lhl1Ks3mU8/3UipUuHExlr338YEk/vuu4+IiIhAh1HgTpw4Qffu3bNNEv4QlJXZy5cn0L//5/z8824Abr65HhMmdKZy5dIBjswYk58iIyPp27dvoMMocOHh4fTr16/Ajhd0ieKHHxK48so3UYVq1coyadJ1dO16SaDDMsWUt8dQjfEHf1QnBF2iaN68Mp061aJZs/MZObI10dH+fyLAmOxERkayf/9+62rcFBh1x6PI70dmi3xl9qZN+xk8eAHjxnXikkucR+syMpSQEPtgmsCyEe5MIOQ0wt25VGYX2TuKlJQ0Xnjhe55//ntSUtKJjAxj1qy/AViSMIVCiRIl8nWUMWMCxa9PPYlIZxH5VUQ2i8gZrVFEJEJEPnCX/yAi1XzZ71dfbaFx42mMGvUtKSnp3HVXU6ZN65bf4RtjjMGPdxQiEgpMBjoACcBKEZmjqus9VrsbOKiqtUSkF/Ai0PPMvZ2y9UBZrr32PwDUq1eBadO6WSd+xhjjR/68o2gObFbVLap6ApgJ3JhlnRuBf7uvZwHtJZdav4PHooiMDOO559qxenV/SxLGGONnfqvMFpFbgM6q+g93ui9whao+4LHOL+46Ce707+46+7Ls6x7gZMfwDYFf/BJ00VMB2JfrWsWDXYtT7FqcYtfilDqqelbdzPqzMju7O4OsWcmXdVDV14DXAEQk/mxr7oONXYtT7FqcYtfiFLsWp4hI/Nlu68+ipwTgIo/pKsCfOa0jImFAGeCAH2MyxhiTR/5MFCuB2iJSXUTCgV7AnCzrzAHucF/fAnytRa1hhzHGBDm/FT2papqIPAAsAEKBt1R1nYiMxhnkew7wJvAfEdmMcyfRy4ddv+avmIsguxan2LU4xa7FKXYtTjnra1HkWmYbY4wpWEHbzbgxxpj8YYnCGGOMV4U2Ufir+4+iyIdrMURE1ovIGhH5SkSCthVibtfCY71bRERFJGgfjfTlWojI39z3xjoRea+gYywoPnxGqorIYhH5yf2cFMwYogVMRN4SkT1uG7XslouITHSv0xoRudSnHatqofvDqfz+HagBhAM/A/WzrDMAmOa+7gV8EOi4A3gtrgGi3df3Fedr4a4XAywBlgNxgY47gO+L2sBPQDl3+rxAxx3Aa/EacJ/7uj6wLdBx++latAYuBX7JYXkXYB5OG7YWwA++7Lew3lH4pfuPIirXa6Gqi1X1mDu5HKfNSjDy5X0B8DQwBgjm/r19uRb/BCar6kEAVd1TwDEWFF+uhQInh7gsw5ltuoKCqi7Be1u0G4F31LEcKCsiF+S238KaKCoDOzymE9x52a6jqmlAIlC+QKIrWL5cC0934/xiCEa5XgsRaQZcpKqfF2RgAeDL++IS4BIRWSoiy0Wkc4FFV7B8uRajgNtFJAGYCzxYMKEVOnn9PgEK73gU+db9RxDw+TxF5HYgDmjj14gCx+u1EJEQYDxwZ0EFFEC+vC/CcIqf2uLcZX4nIg1V9ZCfYytovlyL3sAMVX1ZRFritN9qqKoZ/g+vUDmr783Cekdh3X+c4su1QESuBUYAN6hqSgHFVtByuxYxOJ1GfiMi23DKYOcEaYW2r5+Rz1Q1VVW3Ar/iJI5g48u1uBv4EEBVlwGROB0GFjc+fZ9kVVgThXX/cUqu18ItbpmOkySCtRwacrkWqpqoqhVUtZqqVsOpr7lBVc+6M7RCzJfPyKc4DzogIhVwiqK2FGiUBcOXa7EdaA8gIvVwEsXeAo2ycJgD9HOffmoBJKrqrtw2KpRFT+q/7j+KHB+vxUtAKeAjtz5/u6reELCg/cTHa1Es+HgtFgAdRWQ9kA4MVdX9gYvaP3y8Fg8Dr4vIYJyiljuD8YeliLyPU9RYwa2PeRIoAaCq03DqZ7oAm4FjwF0+7TcIr5Uxxph8VFiLnowxxhQSliiMMcZ4ZYnCGGOMV5YojDHGeGWJwhhjjFeWKEyhIyLpIrLa46+al3Wr5dRTZh6P+Y3b++jPbpcXdc5iH/1FpJ/7+k4RudBj2RsiUj+f41wpIk192GaQiESf67FN8WWJwhRGx1W1qcfftgI6bh9VbYLT2eRLed1YVaep6jvu5J3AhR7L/qGq6/MlylNxTsG3OAcBlijMWbNEYYoE987hOxH50f27Mpt1GojICvcuZI2I1Hbn3+4xf7qIhOZyuCVALXfb9u4YBmvdvv4j3PkvyKkxQMa680aJyCMicgtOn1v/5x4zyr0TiBOR+0RkjEfMd4rIq2cZ5zI8OnQTkakiEi/O2BNPufMG4iSsxSKy2J3XUUSWudfxIxEplctxTDFnicIURlEexU6z3Xl7gA6qeinQE5iYzXb9gQmq2hTnizrB7a6hJ9DKnZ8O9Mnl+NcDa0UkEpgB9FTVRjg9GdwnIrHATUADVW0MPOO5sarOAuJxfvk3VdXjHotnAT08pnsCH5xlnJ1xuuk4aYSqxgGNgTYi0lhVJ+L05XONql7jduUxErjWvZbxwJBcjmOKuULZhYcp9o67X5aeSgCT3DL5dJx+i7JaBowQkSrAJ6q6SUTaA5cBK93uTaJwkk52/k9EjgPbcLqhrgNsVdXf3OX/Bu4HJuGMdfGGiHwB+NyluaruFZEtbj87m9xjLHX3m5c4S+J0V+E5QtnfROQenM/1BTgD9KzJsm0Ld/5S9zjhONfNmBxZojBFxWBgN9AE5074jEGJVPU9EfkB6AosEJF/4HSr/G9VHe7DMfp4diAoItmOb+L2LdQcp5O5XsADQLs8nMsHwN+AjcBsVVVxvrV9jhNnFLcXgMlADxGpDjwCXK6qB0VkBk7Hd1kJsFBVe+chXlPMWdGTKSrKALvc8QP64vyaPo2I1AC2uMUtc3CKYL4CbhGR89x1YsX3McU3AtVEpJY73Rf41i3TL6Oqc3EqirN78igJp9vz7HwCdMcZI+EDd16e4lTVVJwipBZusVVp4CiQKCKVgOtyiGU50OrkOYlItIhkd3dmTCZLFKaomALcISLLcYqdjmazTk/gFxFZDdTFGfJxPc4X6pcisgZYiFMskytVTcbpXfMjEVkLZADTcL50P3f39y3O3U5WM4BpJyuzs+z3ILAeuFhVV7jz8hynW/fxMvCIqv6MMz72OuAtnOKsk14D5onIYlXdi/NE1vvucZbjXCtjcmS9xxpjjPHK7iiMMcZ4ZYnCGGOMV5YojDHGeGWJwhhjjFeWKIwxxnhlicIYY4xXliiMMcZ49f+t84JzzoTh4AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import cycle\n",
    "\n",
    "from sklearn import svm, datasets\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from scipy import interp\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "# Import some data to play with\n",
    "iris = datasets.load_iris()\n",
    "X = iris.data\n",
    "y = iris.target\n",
    "\n",
    "# Binarize the output\n",
    "y = label_binarize(y, classes=[0, 1, 2])\n",
    "n_classes = y.shape[1]\n",
    "\n",
    "# Add noisy features to make the problem harder\n",
    "random_state = np.random.RandomState(0)\n",
    "n_samples, n_features = X.shape\n",
    "X = np.c_[X, random_state.randn(n_samples, 200 * n_features)]\n",
    "\n",
    "# shuffle and split training and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.5,random_state=0)\n",
    "\n",
    "# Learn to predict each class against the other\n",
    "classifier = OneVsRestClassifier(svm.SVC(kernel='linear', probability=True,\n",
    "                                 random_state=random_state))\n",
    "y_score = classifier.fit(X_train, y_train).decision_function(X_test)\n",
    "\n",
    "# Compute the prob\n",
    "y_prob = classifier.predict_proba(X_test)\n",
    "#print(y_prob)\n",
    "\n",
    "# Compute ROC curve and ROC area for each class\n",
    "fpr = dict()\n",
    "tpr = dict()\n",
    "roc_auc = dict()\n",
    "for i in range(n_classes):\n",
    "    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])\n",
    "    roc_auc[i] = auc(fpr[i], tpr[i])\n",
    "\n",
    "# Compute micro-average ROC curve and ROC area\n",
    "fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel())\n",
    "roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])\n",
    "\n",
    "plt.figure()\n",
    "lw = 2\n",
    "plt.plot(fpr[2], tpr[2], color='darkorange',\n",
    "         lw=lw, label='ROC curve (area = %0.2f)' % roc_auc[2])\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic example')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 几个回归的函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9571734475374732"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import explained_variance_score\n",
    "y_true = [3, -0.5, 2, 7]\n",
    "y_pred = [2.5, 0.0, 2, 8]\n",
    "explained_variance_score(y_true, y_pred)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import mean_absolute_error\n",
    "y_true = [3, -0.5, 2, 7]\n",
    "y_pred = [2.5, 0.0, 2, 8]\n",
    "mean_absolute_error(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.375"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "y_true = [3, -0.5, 2, 7]\n",
    "y_pred = [2.5, 0.0, 2, 8]\n",
    "mean_squared_error(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9486081370449679"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " from sklearn.metrics import r2_score\n",
    " y_true = [3, -0.5, 2, 7]\n",
    " y_pred = [2.5, 0.0, 2, 8]\n",
    " r2_score(y_true, y_pred)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
